# -*- coding: utf-8 -*-
import re
import time

import scrapy
from scrapy.http import Request


class HouseSpider(scrapy.Spider):

    name = 'house'
    base_url = 'http://bd.cityhouse.cn'
    allowed_domains = ['http://bd.cityhouse.cn']
    # start_urls = ['http://http://bd.cityhouse.cn/']
    start_urls = ['http://bd.cityhouse.cn/information/hannewlist/GB.html']

    def parse(self, response):
        # print(response.xpath('//div[@class="street_wrap"]/ul/li/a/@href').extract())

        urls = response.xpath('//div[@class="street_wrap"]/ul/li')
        # t = []
        k = 0
        for url in urls:
            k += 1
            # print(url)
            # print(k)
            if k % 10 == 0:
                time.sleep(3)
            # data = {
            #     'url': self.base_url + url.xpath('a/@href').extract_first(),
            #     'title': url.xpath('a/@title').extract_first()
            # }
            url = self.base_url + url.xpath('a/@href').extract_first()
            # print(url_response)
            # return
            # print(url.xpath('a/@href').extract_first())
            # t.append(data)
            yield Request(url, callback=self.getInfo, dont_filter=True)
        # print(t)
        # print(response.body.decode('utf8'))

    def getInfo(self, response):
        # city_search = re.compile(r'<iframe name="fenbuiframe" src="(.*?)">(.*?)</iframe>')

        # 使用正则表达式获取城市列表
        # self.city_list = set(city_search.findall(response.body))
        # print(city_search.findall(response.text))
        # print(response.url)
        domain = response.url.split("/")[-2]
        filename = '%s.html' %domain
        with open(filename, 'wb') as f:
            f.write(response.body)
        # textlist_no_scripts = response.selector.xpath('//*[not(self::script or self::style)] / text()[normalize - space(.)]').extract() textlist_with_scripts =
        # response.selector.xpath('//text()[normalize-space(.)]').extract()
        # with open('filename_no_scripts', 'w', encoding='utf-8') as f:
        # with open('filename_with_scripts', 'w', encoding='utf-8') as f:
        #     for i in range(0, len(textlist_with_scripts)):
        #         text = textlist_with_scripts[i].strip()
        #         f.write(text + '\n')
        print('---------------------------------------------------')